/*
 *  This file is part of VADER.
 *
 *  VADER is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  VADER is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package vegadataeditor;

import java.io.IOException;
import org.jdesktop.application.Action;
import org.jdesktop.application.ResourceMap;
import org.jdesktop.application.SingleFrameApplication;
import org.jdesktop.application.FrameView;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import javax.swing.Timer;
import javax.swing.Icon;
import javax.swing.JDialog;
import javax.swing.JFrame;
import java.sql.ResultSet;
import java.sql.Statement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import javax.swing.SwingWorker;
import java.beans.PropertyChangeEvent;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashSet;
import java.io.FileWriter;
import java.io.BufferedWriter;
import java.io.FileReader;
import javax.swing.JFileChooser;
import javax.swing.filechooser.FileNameExtensionFilter;
import au.com.bytecode.opencsv.CSVReader;
import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent;
import java.awt.event.WindowListener;
import java.util.regex.Pattern;
import java.util.prefs.Preferences;
import java.util.regex.Matcher;
import javax.swing.event.DocumentListener;
import javax.swing.event.DocumentEvent;

import org.jfree.chart.*;
import org.jfree.data.xy.*;

import vegadataeditor.output.dateFormaters.*;
import vegadataeditor.output.dataAggregators.*;

import vegadataeditor.dataSources.GleonWebServiceSource;
import vegadataeditor.dataSources.IDataSource;





/**
 * The main UI view for the VADER application.
 *
 * @author lawinslow
 */
public class VegaDataEditorView
        extends FrameView
        implements java.beans.PropertyChangeListener,ActionListener{

    /**
     * Netbeans added variables to deal with animations and other UI items.
     * 2009-2-20: I changed them from final.
     */
    private final Timer messageTimer;
    private final Timer busyIconTimer;
    private final Icon idleIcon;
    private final Icon[] busyIcons = new Icon[15];
    private int busyIconIndex = 0;
    private JDialog aboutBox;


    /** The current data source object. */
    private IDataSource source = new GleonWebServiceSource();
    /** All stream metadata loaded. Used as master copy. */
    private List<VegaStreamContainer> allStreamInfo;
    /** Currently displayed stream objects. Not all streams due to filtering. */
    private List<VegaStreamContainer> shownStreams;
    /** Streams that have been added to the selected list box. */
    private List<VegaStreamContainer> selectedStreams
            = new ArrayList<VegaStreamContainer>();
    /** File chooser used when saving and opening output files. */
    private JFileChooser fileChooser = new JFileChooser();
    /** The dataset plotted in the JFreeChart plot window on the main view. */
    private DefaultXYDataset dataset = new DefaultXYDataset();
    /** Reference to the preview chart on the main view. */
    private JFreeChart chart;
    /** Reference to the last operation performed. Used to indicate if data
     * are to be plotted or output to file after web service returns from
     * async call.*/
    private String lastOperation = "unknown";
    /** Standard date format used to parse and format all dates on the interface. */
    private String dateFormat = "yyyy/MM/dd";
    /** Preferences node to store minor program settings */
    private Preferences prefs =
            Preferences.userNodeForPackage(VegaDataEditorView.class);
    /**
     * Main constructor
     * @param app launching application.
     */
    public VegaDataEditorView(SingleFrameApplication app) {
        super(app);
        initComponents();
        setupCloseListener();
        setupPreviewPane();
        setupFilterListeners();
    

        /***********************************************************************
         * All of this stuff needs to be in the constructor because they are 
         * final variables.
         **********************************************************************/
        // status bar initialization - message timeout, idle icon and busy animation, etc
        ResourceMap resourceMap = getResourceMap();
        int messageTimeout = resourceMap.getInteger("StatusBar.messageTimeout");
        messageTimer = new Timer(messageTimeout, new ActionListener() {
            public void actionPerformed(ActionEvent e) {
                statusMessageLabel.setText("");
            }
        });    
        
        messageTimer.setRepeats(false);
        int busyAnimationRate = resourceMap.getInteger("StatusBar.busyAnimationRate");
        for (int i = 0; i < busyIcons.length; i++) {
            busyIcons[i] = resourceMap.getIcon("StatusBar.busyIcons[" + i + "]");
        }
        busyIconTimer = new Timer(busyAnimationRate, new ActionListener() {
            public void actionPerformed(ActionEvent e) {
                busyIconIndex = (busyIconIndex + 1) % busyIcons.length;
                statusAnimationLabel.setIcon(busyIcons[busyIconIndex]);
            }
        });
        idleIcon = resourceMap.getIcon("StatusBar.idleIcon");
        statusAnimationLabel.setIcon(idleIcon);
        progressBar.setVisible(false);
        

        //We'll use a single file chooser for all operations. Right now just CSV
        FileNameExtensionFilter csvFilter = new FileNameExtensionFilter("CSV File","csv");
        fileChooser.setFileFilter(csvFilter);

        //Get stored start/end date preferences. Setup store change events
        startDateTF.setText(prefs.get("startDate", ""));
        endDateTF.setText(prefs.get("endDate",""));

        endDateTF.getDocument().addDocumentListener(new DocumentListener(){
            public void insertUpdate(DocumentEvent e) {
                prefs.put("endDate", endDateTF.getText());
            }
            public void removeUpdate(DocumentEvent e) {
                prefs.put("endDate", endDateTF.getText());
            }
            public void changedUpdate(DocumentEvent e) {
                prefs.put("endDate", endDateTF.getText());
            }
        });
        startDateTF.getDocument().addDocumentListener(new DocumentListener(){
            public void insertUpdate(DocumentEvent e) {
                prefs.put("startDate", startDateTF.getText());
            }
            public void removeUpdate(DocumentEvent e) {
                prefs.put("startDate", startDateTF.getText());
            }
            public void changedUpdate(DocumentEvent e) {
                prefs.put("startDate", startDateTF.getText());
            }
        });


    }//VegaDataEditor

    /**
     * Sets up proper method call on close. We need to disconnect from db
     * and cleanup first, otherwise the program hangs.
     */
    private void setupCloseListener(){
        JFrame fatherFrame = this.getFrame();

        fatherFrame.setDefaultCloseOperation ( JFrame. DO_NOTHING_ON_CLOSE );
        WindowListener windowListener = new WindowAdapter()
           {

            @Override
           public void windowClosing ( WindowEvent w )
              {
                cleanupAndClose();
              }
           };

        fatherFrame.addWindowListener( windowListener );
    }

    /**
     * Creates and properly adds the chart panel to the main UI. Using
     * the TimeSeriesChart of JFreeChart
     */
    private void setupPreviewPane(){
                chart = ChartFactory.createTimeSeriesChart(null,
                null,
                null,
                dataset, false, false, false);

        ChartPanel cpanel = new ChartPanel(chart,true);
        cpanel.setDisplayToolTips(true);
        cpanel.setToolTipText("<html>Click and drag to zoom in." +
                " <br> Right click and select autorange" +
                " <br> both axes to return to full range.");


        chartPanel.add(cpanel,"graph");
    }

    /**
     * Setup listeners for filter change events to update the streams list
     * when user changes selection.
     */
    private void setupFilterListeners(){

        sitesCB.addActionListener(this);
        variablesCB.addActionListener(this);

        DocumentListener l = new DocumentListener(){
            public void changedUpdate(DocumentEvent e){
                filterStreams();
            }
            public void removeUpdate(DocumentEvent e){
                filterStreams();
            }
            public void insertUpdate(DocumentEvent e){
                filterStreams();
            }
        };

        filterField.getDocument().addDocumentListener(l);
    }

    /**
     * Property change method that handles changes in async process status.
     * Starts, stops, and updates progress bar.
     * @param evt
     */
    public void propertyChange(PropertyChangeEvent evt) {
        String propertyName = evt.getPropertyName();
        if ("started".equals(propertyName)
                ||("state".equals(propertyName) && 
                evt.getNewValue().equals(SwingWorker.StateValue.STARTED))) {
            if (!busyIconTimer.isRunning()) {
                statusAnimationLabel.setIcon(busyIcons[0]);
                busyIconIndex = 0;
                busyIconTimer.start();
            }
            progressBar.setVisible(true);
            progressBar.setIndeterminate(true);
            //Added the state check as SwingWorker doesn't send
            // done and started, rather state with a value of started and done.
        } else if ("done".equals(propertyName)||
                ("state".equals(propertyName) && 
                evt.getNewValue().equals(SwingWorker.StateValue.DONE))) {
            busyIconTimer.stop();
            statusAnimationLabel.setIcon(idleIcon);
            progressBar.setVisible(false);
            progressBar.setValue(0);
        } else if ("message".equals(propertyName)) {
            String text = (String)(evt.getNewValue());
            statusMessageLabel.setText((text == null) ? "" : text);
            messageTimer.restart();
        } else if ("progress".equals(propertyName)) {
            int value = (Integer)(evt.getNewValue());
            progressBar.setVisible(true);
            progressBar.setIndeterminate(false);
            progressBar.setValue(value);
        }
    }
    
    /**
     * Displays the about box dialog frame.
     */
    @Action
    public void showAboutBox() {
        if (aboutBox == null) {
            JFrame mainFrame = VegaDataEditorApp.getApplication().getMainFrame();
            aboutBox = new VegaDataEditorAboutBox(mainFrame);
            aboutBox.setLocationRelativeTo(mainFrame);
        }
        VegaDataEditorApp.getApplication().show(aboutBox);
    }


    /** This method is called from within the constructor to
     * initialize the form.
     * WARNING: Do NOT modify this code. The content of this method is
     * always regenerated by the Form Editor.
     */
    // <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
    private void initComponents() {

        mainPanel = new javax.swing.JPanel();
        chartPanel = new javax.swing.JPanel();
        jPanel1 = new javax.swing.JPanel();
        sitesCB = new javax.swing.JComboBox();
        jLabel3 = new javax.swing.JLabel();
        filterField = new javax.swing.JTextField();
        variablesCB = new javax.swing.JComboBox();
        jLabel4 = new javax.swing.JLabel();
        jLabel5 = new javax.swing.JLabel();
        jScrollPane1 = new javax.swing.JScrollPane();
        streamsList = new javax.swing.JList();
        removeButton = new javax.swing.JButton();
        jScrollPane2 = new javax.swing.JScrollPane();
        selectedStreamsList = new javax.swing.JList();
        jPanel3 = new javax.swing.JPanel();
        jLabel1 = new javax.swing.JLabel();
        jLabel2 = new javax.swing.JLabel();
        startDateTF = new javax.swing.JTextField();
        endDateTF = new javax.swing.JTextField();
        toCSVButton = new javax.swing.JButton();
        outputDateFormatCB = new javax.swing.JComboBox();
        fileTypeLabel = new javax.swing.JLabel();
        previewButton = new javax.swing.JButton();
        aggTypeCB = new javax.swing.JComboBox();
        jLabel8 = new javax.swing.JLabel();
        aggSpanTF = new javax.swing.JTextField();
        addButton = new javax.swing.JButton();
        jPanel2 = new javax.swing.JPanel();
        metadataLabel = new javax.swing.JLabel();
        jLabel6 = new javax.swing.JLabel();
        jLabel7 = new javax.swing.JLabel();
        menuBar = new javax.swing.JMenuBar();
        javax.swing.JMenu fileMenu = new javax.swing.JMenu();
        javax.swing.JMenuItem exitMenuItem = new javax.swing.JMenuItem();
        javax.swing.JMenu helpMenu = new javax.swing.JMenu();
        javax.swing.JMenuItem aboutMenuItem = new javax.swing.JMenuItem();
        statusPanel = new javax.swing.JPanel();
        javax.swing.JSeparator statusPanelSeparator = new javax.swing.JSeparator();
        statusMessageLabel = new javax.swing.JLabel();
        statusAnimationLabel = new javax.swing.JLabel();
        progressBar = new javax.swing.JProgressBar();

        mainPanel.setName("mainPanel"); // NOI18N

        org.jdesktop.application.ResourceMap resourceMap = org.jdesktop.application.Application.getInstance(vegadataeditor.VegaDataEditorApp.class).getContext().getResourceMap(VegaDataEditorView.class);
        chartPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(resourceMap.getString("chartPanel.border.title"))); // NOI18N
        chartPanel.setToolTipText(resourceMap.getString("chartPanel.toolTipText")); // NOI18N
        chartPanel.setName("chartPanel"); // NOI18N
        chartPanel.setLayout(new java.awt.CardLayout());

        jPanel1.setBorder(javax.swing.BorderFactory.createTitledBorder(resourceMap.getString("jPanel1.border.title"))); // NOI18N
        jPanel1.setName("jPanel1"); // NOI18N

        sitesCB.setName("siteCB"); // NOI18N

        jLabel3.setText(resourceMap.getString("jLabel3.text")); // NOI18N
        jLabel3.setName("jLabel3"); // NOI18N

        filterField.setText(resourceMap.getString("filterField.text")); // NOI18N
        filterField.setName("filterField"); // NOI18N

        variablesCB.setMaximumSize(new java.awt.Dimension(23, 20));
        variablesCB.setName("variablesCB"); // NOI18N

        jLabel4.setText(resourceMap.getString("jLabel4.text")); // NOI18N
        jLabel4.setName("jLabel4"); // NOI18N

        jLabel5.setText(resourceMap.getString("jLabel5.text")); // NOI18N
        jLabel5.setName("jLabel5"); // NOI18N

        javax.swing.GroupLayout jPanel1Layout = new javax.swing.GroupLayout(jPanel1);
        jPanel1.setLayout(jPanel1Layout);
        jPanel1Layout.setHorizontalGroup(
            jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(jPanel1Layout.createSequentialGroup()
                .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addComponent(filterField)
                    .addComponent(variablesCB, 0, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
                    .addComponent(sitesCB, 0, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addComponent(jLabel3)
                    .addComponent(jLabel4)
                    .addComponent(jLabel5)))
        );
        jPanel1Layout.setVerticalGroup(
            jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(jPanel1Layout.createSequentialGroup()
                .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                    .addComponent(sitesCB, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                    .addComponent(jLabel3))
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                    .addComponent(variablesCB, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                    .addComponent(jLabel4))
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                    .addComponent(filterField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                    .addComponent(jLabel5))
                .addContainerGap())
        );

        jScrollPane1.setName("jScrollPane1"); // NOI18N

        streamsList.setName("streamsList"); // NOI18N
        streamsList.addListSelectionListener(new javax.swing.event.ListSelectionListener() {
            public void valueChanged(javax.swing.event.ListSelectionEvent evt) {
                streamsListValueChanged(evt);
            }
        });
        jScrollPane1.setViewportView(streamsList);

        removeButton.setIcon(resourceMap.getIcon("removeButton.icon")); // NOI18N
        removeButton.setText(resourceMap.getString("removeButton.text")); // NOI18N
        removeButton.setName("removeButton"); // NOI18N
        removeButton.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                removeButtonActionPerformed(evt);
            }
        });

        jScrollPane2.setName("jScrollPane2"); // NOI18N

        selectedStreamsList.setToolTipText(resourceMap.getString("selectedStreamsList.toolTipText")); // NOI18N
        selectedStreamsList.setName("selectedStreamsList"); // NOI18N
        selectedStreamsList.addListSelectionListener(new javax.swing.event.ListSelectionListener() {
            public void valueChanged(javax.swing.event.ListSelectionEvent evt) {
                selectedStreamsListValueChanged(evt);
            }
        });
        jScrollPane2.setViewportView(selectedStreamsList);

        jPanel3.setBorder(javax.swing.BorderFactory.createTitledBorder(resourceMap.getString("jPanel3.border.title"))); // NOI18N
        jPanel3.setName("jPanel3"); // NOI18N

        jLabel1.setText(resourceMap.getString("jLabel1.text")); // NOI18N
        jLabel1.setName("jLabel1"); // NOI18N

        jLabel2.setText(resourceMap.getString("jLabel2.text")); // NOI18N
        jLabel2.setName("jLabel2"); // NOI18N

        startDateTF.setText(resourceMap.getString("startDateTF.text")); // NOI18N
        startDateTF.setName("startDateTF"); // NOI18N

        endDateTF.setText(resourceMap.getString("endDateTF.text")); // NOI18N
        endDateTF.setName("endDateTF"); // NOI18N

        toCSVButton.setText(resourceMap.getString("toCSVButton.text")); // NOI18N
        toCSVButton.setToolTipText(resourceMap.getString("toCSVButton.toolTipText")); // NOI18N
        toCSVButton.setName("toCSVButton"); // NOI18N
        toCSVButton.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                toCSVButtonActionPerformed(evt);
            }
        });

        outputDateFormatCB.setModel(new javax.swing.DefaultComboBoxModel(new String[] { "yyyy-MM-dd HH:mm:ss", "MM/dd/yyyy HH:mm:ss", "dd/MM/yyyy HH:mm:ss", "Year,month,day,....", "Year Fraction", "Julian Day" }));
        outputDateFormatCB.setName("outputDateFormatCB"); // NOI18N

        fileTypeLabel.setText(resourceMap.getString("fileTypeLabel.text")); // NOI18N
        fileTypeLabel.setName("fileTypeLabel"); // NOI18N

        previewButton.setText(resourceMap.getString("previewButton.text")); // NOI18N
        previewButton.setToolTipText(resourceMap.getString("previewButton.toolTipText")); // NOI18N
        previewButton.setEnabled(false);
        previewButton.setName("previewButton"); // NOI18N
        previewButton.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                previewButtonActionPerformed(evt);
            }
        });

        aggTypeCB.setModel(new javax.swing.DefaultComboBoxModel(new String[] { "RAW", "MEAN", "MIN", "MAX", "SUM", "STDDEV" }));
        aggTypeCB.setName("aggTypeCB"); // NOI18N
        aggTypeCB.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                aggTypeCBActionPerformed(evt);
            }
        });

        jLabel8.setText(resourceMap.getString("jLabel8.text")); // NOI18N
        jLabel8.setName("jLabel8"); // NOI18N

        aggSpanTF.setText(resourceMap.getString("aggSpanTF.text")); // NOI18N
        aggSpanTF.setName("aggSpanTF"); // NOI18N

        javax.swing.GroupLayout jPanel3Layout = new javax.swing.GroupLayout(jPanel3);
        jPanel3.setLayout(jPanel3Layout);
        jPanel3Layout.setHorizontalGroup(
            jPanel3Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(jPanel3Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false)
                .addComponent(startDateTF)
                .addComponent(jLabel1, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
                .addComponent(jLabel2)
                .addComponent(endDateTF))
            .addComponent(fileTypeLabel)
            .addGroup(jPanel3Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false)
                .addComponent(toCSVButton, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
                .addComponent(previewButton, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
            .addComponent(jLabel8)
            .addGroup(jPanel3Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false)
                .addGroup(jPanel3Layout.createSequentialGroup()
                    .addComponent(aggTypeCB, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                    .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                    .addComponent(aggSpanTF, 0, 0, Short.MAX_VALUE))
                .addComponent(outputDateFormatCB, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.PREFERRED_SIZE, 121, javax.swing.GroupLayout.PREFERRED_SIZE))
        );
        jPanel3Layout.setVerticalGroup(
            jPanel3Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(jPanel3Layout.createSequentialGroup()
                .addComponent(jLabel1)
                .addGap(2, 2, 2)
                .addComponent(startDateTF, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addComponent(jLabel2)
                .addGap(3, 3, 3)
                .addComponent(endDateTF, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addComponent(previewButton)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addComponent(toCSVButton)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addComponent(fileTypeLabel)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addComponent(outputDateFormatCB, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                .addGap(6, 6, 6)
                .addComponent(jLabel8)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addGroup(jPanel3Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                    .addComponent(aggTypeCB, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                    .addComponent(aggSpanTF, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))
                .addContainerGap())
        );

        addButton.setIcon(resourceMap.getIcon("addButton.icon")); // NOI18N
        addButton.setText(resourceMap.getString("addButton.text")); // NOI18N
        addButton.setName("addButton"); // NOI18N
        addButton.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                addButtonActionPerformed(evt);
            }
        });

        jPanel2.setBorder(javax.swing.BorderFactory.createTitledBorder(resourceMap.getString("jPanel2.border.title"))); // NOI18N
        jPanel2.setName("jPanel2"); // NOI18N

        metadataLabel.setText(resourceMap.getString("metadataLabel.text")); // NOI18N
        metadataLabel.setName("metadataLabel"); // NOI18N

        javax.swing.GroupLayout jPanel2Layout = new javax.swing.GroupLayout(jPanel2);
        jPanel2.setLayout(jPanel2Layout);
        jPanel2Layout.setHorizontalGroup(
            jPanel2Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(jPanel2Layout.createSequentialGroup()
                .addComponent(metadataLabel)
                .addContainerGap())
        );
        jPanel2Layout.setVerticalGroup(
            jPanel2Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addComponent(metadataLabel)
        );

        jLabel6.setText(resourceMap.getString("jLabel6.text")); // NOI18N
        jLabel6.setName("jLabel6"); // NOI18N

        jLabel7.setText(resourceMap.getString("jLabel7.text")); // NOI18N
        jLabel7.setName("jLabel7"); // NOI18N

        javax.swing.GroupLayout mainPanelLayout = new javax.swing.GroupLayout(mainPanel);
        mainPanel.setLayout(mainPanelLayout);
        mainPanelLayout.setHorizontalGroup(
            mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(mainPanelLayout.createSequentialGroup()
                .addContainerGap()
                .addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addGroup(mainPanelLayout.createSequentialGroup()
                        .addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false)
                            .addComponent(jPanel2, javax.swing.GroupLayout.PREFERRED_SIZE, 284, javax.swing.GroupLayout.PREFERRED_SIZE)
                            .addComponent(jPanel1, javax.swing.GroupLayout.DEFAULT_SIZE, 284, Short.MAX_VALUE))
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                            .addComponent(jLabel6)
                            .addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 251, Short.MAX_VALUE)))
                    .addComponent(chartPanel, javax.swing.GroupLayout.DEFAULT_SIZE, 541, Short.MAX_VALUE))
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addComponent(addButton, javax.swing.GroupLayout.PREFERRED_SIZE, 40, javax.swing.GroupLayout.PREFERRED_SIZE)
                    .addComponent(removeButton, javax.swing.GroupLayout.PREFERRED_SIZE, 40, javax.swing.GroupLayout.PREFERRED_SIZE))
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addComponent(jLabel7)
                    .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, 280, javax.swing.GroupLayout.PREFERRED_SIZE)
                    .addComponent(jPanel3, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))
                .addContainerGap(15, Short.MAX_VALUE))
        );
        mainPanelLayout.setVerticalGroup(
            mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(mainPanelLayout.createSequentialGroup()
                .addContainerGap()
                .addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addGroup(mainPanelLayout.createSequentialGroup()
                        .addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING)
                            .addComponent(jLabel6)
                            .addComponent(jLabel7))
                        .addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING)
                            .addGroup(mainPanelLayout.createSequentialGroup()
                                .addGap(6, 6, 6)
                                .addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                                    .addGroup(mainPanelLayout.createSequentialGroup()
                                        .addGap(22, 22, 22)
                                        .addComponent(addButton)
                                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                                        .addComponent(removeButton))
                                    .addComponent(jScrollPane2, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, 234, Short.MAX_VALUE)))
                            .addGroup(mainPanelLayout.createSequentialGroup()
                                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                                .addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 234, Short.MAX_VALUE))))
                    .addGroup(mainPanelLayout.createSequentialGroup()
                        .addComponent(jPanel1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(jPanel2, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, 35, Short.MAX_VALUE)))
                .addGap(0, 0, 0)
                .addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addComponent(chartPanel, javax.swing.GroupLayout.DEFAULT_SIZE, 275, Short.MAX_VALUE)
                    .addComponent(jPanel3, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))
                .addContainerGap())
        );

        menuBar.setName("menuBar"); // NOI18N

        fileMenu.setText(resourceMap.getString("fileMenu.text")); // NOI18N
        fileMenu.setName("fileMenu"); // NOI18N

        javax.swing.ActionMap actionMap = org.jdesktop.application.Application.getInstance(vegadataeditor.VegaDataEditorApp.class).getContext().getActionMap(VegaDataEditorView.class, this);
        exitMenuItem.setAction(actionMap.get("quit")); // NOI18N
        exitMenuItem.setName("exitMenuItem"); // NOI18N
        exitMenuItem.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                exitMenuItemActionPerformed(evt);
            }
        });
        fileMenu.add(exitMenuItem);

        menuBar.add(fileMenu);

        helpMenu.setText(resourceMap.getString("helpMenu.text")); // NOI18N
        helpMenu.setName("helpMenu"); // NOI18N

        aboutMenuItem.setAction(actionMap.get("showAboutBox")); // NOI18N
        aboutMenuItem.setName("aboutMenuItem"); // NOI18N
        helpMenu.add(aboutMenuItem);

        menuBar.add(helpMenu);

        statusPanel.setName("statusPanel"); // NOI18N

        statusPanelSeparator.setName("statusPanelSeparator"); // NOI18N

        statusMessageLabel.setName("statusMessageLabel"); // NOI18N

        statusAnimationLabel.setHorizontalAlignment(javax.swing.SwingConstants.LEFT);
        statusAnimationLabel.setName("statusAnimationLabel"); // NOI18N

        progressBar.setName("progressBar"); // NOI18N

        javax.swing.GroupLayout statusPanelLayout = new javax.swing.GroupLayout(statusPanel);
        statusPanel.setLayout(statusPanelLayout);
        statusPanelLayout.setHorizontalGroup(
            statusPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(statusPanelLayout.createSequentialGroup()
                .addContainerGap()
                .addComponent(statusMessageLabel)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, 728, Short.MAX_VALUE)
                .addComponent(progressBar, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addComponent(statusAnimationLabel)
                .addContainerGap())
            .addComponent(statusPanelSeparator, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, 898, Short.MAX_VALUE)
        );
        statusPanelLayout.setVerticalGroup(
            statusPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, statusPanelLayout.createSequentialGroup()
                .addComponent(statusPanelSeparator, javax.swing.GroupLayout.PREFERRED_SIZE, 2, javax.swing.GroupLayout.PREFERRED_SIZE)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
                .addGroup(statusPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                    .addComponent(statusMessageLabel)
                    .addComponent(statusAnimationLabel)
                    .addComponent(progressBar, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))
                .addGap(3, 3, 3))
        );

        setComponent(mainPanel);
        setMenuBar(menuBar);
        setStatusBar(statusPanel);
    }// </editor-fold>//GEN-END:initComponents


    /**
     * Fills the sites filter combo box with the contents of the supplied list.
     * 
     * @param sites A list of strings
     */
    public void setSites(List<String> sites){

            sitesCB.removeAllItems();

            sitesCB.addItem("");

            for(String s:sites){
                sitesCB.addItem(s);
            }
            sitesCB.updateUI();

    }

    public void setVariables(List<String> variables){
        variablesCB.removeAllItems();

        variablesCB.addItem("");

        for(String v:variables){
            variablesCB.addItem(v);
        }
        variablesCB.updateUI();

    }

    public void setStreams(List<VegaStreamContainer> streams){
        allStreamInfoRetrieved(streams);
    }

    
    
    private void exitMenuItemActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_exitMenuItemActionPerformed
        cleanupAndClose();
    }//GEN-LAST:event_exitMenuItemActionPerformed

    private void cleanupAndClose(){
        try{
            prefs.flush();
        } catch (Exception e){
            //Do nothing
        }
        System.exit(0);
    }

    private void toCSVButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_toCSVButtonActionPerformed

        //We want save button to save all streams in the selected streams list
        lastOperation = "save";
        startPopulateSelectedStreams(selectedStreams);
    }//GEN-LAST:event_toCSVButtonActionPerformed

    private void uploadChanges(){
        if(fileChooser.showOpenDialog(mainPanel)!=JFileChooser.APPROVE_OPTION){
            return;
        }
        
        java.io.File toRead = fileChooser.getSelectedFile();
        
        try{
            CSVReader br = new CSVReader(new FileReader(toRead));
            SparseMatrixContainer toParse = readUploadedFile(br);
            
            UpdateValues toRun = new UpdateValues(toParse);
            toRun.addPropertyChangeListener(this);
            
            toRun.execute();
            
        }catch(Exception e){
            e.printStackTrace();
        }
                
    }
    
    private void previewButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_previewButtonActionPerformed
        ArrayList<VegaStreamContainer> toPreview = new ArrayList<VegaStreamContainer>();

        for(int i:selectedStreamsList.getSelectedIndices()){
            toPreview.add(selectedStreams.get(i));
        }

        lastOperation = "plotselected";
        startPopulateSelectedStreams(toPreview);
        
}//GEN-LAST:event_previewButtonActionPerformed

    private void removeButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_removeButtonActionPerformed
        ArrayList<VegaStreamContainer> selected =
            new ArrayList<VegaStreamContainer>();
        int[] indexes = selectedStreamsList.getSelectedIndices();

        VegaStreamContainer tmp = null;
        
        for(int i=0;i<indexes.length;i++){
            tmp = selectedStreams.get(indexes[i]);
            tmp.selected = false;
            selected.add(tmp);
        }
        selectedStreams.removeAll(selected);
        selectedStreamsList.setListData(selectedStreams.toArray(new VegaStreamContainer[0]));
        this.filterStreams();
    }//GEN-LAST:event_removeButtonActionPerformed

    private void addButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_addButtonActionPerformed

        int[] indexes = streamsList.getSelectedIndices();

        VegaStreamContainer tmp = null;
        for(int i=0;i<indexes.length;i++){
            tmp = shownStreams.get(indexes[i]);
            //store the original index so it can be later added back in the same place
            tmp.selected = true;
            selectedStreams.add(tmp);
        }
        


        filterStreams();
        selectedStreamsList.setListData(selectedStreams.toArray(new VegaStreamContainer[0]));
        
    }//GEN-LAST:event_addButtonActionPerformed

    private void selectedStreamsListValueChanged(javax.swing.event.ListSelectionEvent evt) {//GEN-FIRST:event_selectedStreamsListValueChanged
        if(selectedStreamsList.getSelectedIndex() < 0){//then nothing is selected
            previewButton.setEnabled(false);
            return;
        }
        previewButton.setEnabled(true);
        displayStreamMetadata(selectedStreams.get(selectedStreamsList.getSelectedIndex()));
    }//GEN-LAST:event_selectedStreamsListValueChanged

    private void streamsListValueChanged(javax.swing.event.ListSelectionEvent evt) {//GEN-FIRST:event_streamsListValueChanged
        if(streamsList.getSelectedIndex() < 0){//then nothing is selected
            return;
        }
        displayStreamMetadata(shownStreams.get(streamsList.getSelectedIndex()));
    }//GEN-LAST:event_streamsListValueChanged

    private void aggTypeCBActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_aggTypeCBActionPerformed
        if(((String)aggTypeCB.getSelectedItem()).equalsIgnoreCase("RAW")){
            aggSpanTF.setEnabled(false);
        }else{
            aggSpanTF.setEnabled(true);

        }
    }//GEN-LAST:event_aggTypeCBActionPerformed

    private void displayStreamMetadata(VegaStreamContainer stream){
        if(stream == null){
            //put up empty metadata labels
            metadataLabel.setText(  "<html>Site:" + stream.site +
                        "<br />Variable:" +
                        "<br />Unit:" +
                        "<br />Start Date:" +
                        "<br />End Date:");
        }

        //Add hour:minute to the display format
        SimpleDateFormat format = new SimpleDateFormat(dateFormat + " hh:MM");
        try{
            metadataLabel.setText(  "<html>Site:" + stream.site +
                                    "<br />Variable:" + stream.variable +
                                    "<br />Unit:" + stream.unit +
                                    "<br />Start Date:" + format.format(stream.earliestDate) +
                                    "<br />End Date:" + format.format(stream.latestDate));
        }catch(NullPointerException npe){
            metadataLabel.setText(  "<html>Site:" + stream.site +
                                    "<br />Variable:" + stream.variable +
                                    "<br />Unit:" + stream.unit +
                                    "<br />Data Stream is Empty" +
                                    "<br />");
        }

    }

    private void dbUpdateDone(int inserted, int deleted, int updated, int ignored){
        
        javax.swing.JOptionPane.showMessageDialog(this.mainPanel, "Updated:" + updated + 
                " Deleted:" + deleted + " Inserted:" + inserted + " Ignored: " + ignored );
        
    }
            
    private SparseMatrixContainer readUploadedFile(CSVReader br) throws IOException,java.text.ParseException{

        SimpleDateFormat simp = new SimpleDateFormat("MM/dd/yyyy HH:mm");
        SparseMatrixContainer toReturn = new SparseMatrixContainer();
        
        List<String[]> allLines = br.readAll();
        
        double[][] values = new double[allLines.size()-1][];
        Date[] datetime = new Date[allLines.size()-1];
        
        toReturn.headers = allLines.get(0);
        int numColumns = toReturn.headers.length;
        
        for(int i=0;i<allLines.size()-1;i++){
            datetime[i] = simp.parse(allLines.get(i+1)[0]);
            
            values[i] = new double[numColumns -1];
            java.util.Arrays.fill(values[i], Double.NaN);
            
            for(int j=1;j<allLines.get(i+1).length;j++){
                if(allLines.get(i+1)[j].trim().compareTo("")!=0){
                    values[i][j-1] = Double.parseDouble(allLines.get(i+1)[j]);
                }
            }
        }
        
        toReturn.datetime = datetime;
        toReturn.values = values;
        
        return toReturn;
    }
    
    private SparseMatrixContainer createSparseMatrix(List<VegaStreamContainer> streamContainers,ITimeModifier agg){
        if(agg==null){
            agg = new NullModifier();
        }
        
        ArrayList<ModifierResult> streams = new ArrayList<ModifierResult>();
        for(VegaStreamContainer stream:streamContainers)
            streams.add(agg.modify(stream));

        Date start = new Date();
        HashSet<Date> uniqueDateTime = 
                new HashSet<Date>(streams.get(streams.size()-1).getValues().size());
        
        for(ModifierResult stream:streams){
            uniqueDateTime.addAll(stream.getTimes());
        }
        
        java.util.TreeSet<Date> orderedDateTime = 
                new java.util.TreeSet<Date>();
        
        orderedDateTime.addAll(uniqueDateTime);
        
        Date[] allDates = orderedDateTime.toArray(new Date[1]);
        double[][] allData = new double[uniqueDateTime.size()][];
        
        int streamCount = streams.size();
        Date tmpDate;
        int tmpInt;

        int[] valueIndexes = new int[streamCount];
        java.util.Arrays.fill(valueIndexes,0);
        
        for(int i=0;i<allDates.length;i++){
                allData[i] = new double[streamCount];
                java.util.Arrays.fill(allData[i], Double.NaN);
                tmpDate = allDates[i];
                for(int j=0;j<streamCount;j++){
                    if(streams.get(j).getTimes().size() !=0 &&
                            streams.get(j).getTimes().get(valueIndexes[j]).compareTo(tmpDate)==0){
                        allData[i][j] = streams.get(j).getValues().get(valueIndexes[j]);
                        
                        if(valueIndexes[j]+1 < streams.get(j).getValues().size()){
                            valueIndexes[j]++;
                        }
                    }
                }
        }

        SparseMatrixContainer result = new SparseMatrixContainer();
        result.datetime = allDates;
        result.values = allData;
        
        String[] header = new String[streams.size()];
        for(int i=0;i<header.length;i++)
            header[i] = streamContainers.get(i).toString() + "streamID:" + streamContainers.get(i).streamID;
        result.headers = header;
        return result;
    }
    
    private void saveAsCsv(SparseMatrixContainer matrix,IDateFormatter format){
        BufferedWriter bw = null;
        
        try{
            String delim = ",";
            String newline = System.getProperty("line.separator");

           
            int state = fileChooser.showSaveDialog(mainPanel);
            if(state != JFileChooser.APPROVE_OPTION)
                return;

            java.io.File file = fileChooser.getSelectedFile();
            
            String Name = file.getName();


            //or if file does not have suffix, we need to add .csv
            if(!file.getName().trim().endsWith(".csv")){
                //just add .csv
                file = new java.io.File(file.getAbsolutePath().trim() + ".csv");
            }
            
        
            bw = new BufferedWriter(new FileWriter(file));

            bw.write(format.getHeader() + delim);
            for(int i=0;i<matrix.headers.length;i++){
                bw.write(matrix.headers[i]);
                if(i!= matrix.headers.length -1){
                    bw.write(delim);
                }
            }
            bw.write(newline);
            
            for(int i=0;i<matrix.datetime.length;i++){
                bw.write(format.format(matrix.datetime[i]));
                bw.write(delim);
                for(int j=0;j<matrix.values[i].length;j++){
                    if(!Double.isNaN(matrix.values[i][j])){
                        bw.write(Double.toString(matrix.values[i][j]));
                    }
                    if(j!=matrix.values[i].length-1)
                        bw.write(delim);
                }
                bw.write(newline);
            }
            
            
        }catch(java.io.IOException e){
            e.printStackTrace();
            return;
            
        }finally{
            if(bw != null){
                try {
                    bw.flush();
                    bw.close();
                } catch (IOException ignore) {
                }
            }

                    
        }
    }
    
    private void startPopulateSelectedStreams(List<VegaStreamContainer> selected){
        SimpleDateFormat simp = new SimpleDateFormat(dateFormat);
        Date end = new Date();
        Date start = new Date();
        boolean parseException = false;
        try{
            start = simp.parse(startDateTF.getText());
            end = simp.parse(endDateTF.getText());
        }catch(java.text.ParseException pe){
            parseException = true;
        }
        
        if(parseException){
            javax.swing.JOptionPane.showMessageDialog(mainPanel, 
                    "Please check your date selection format.");
            return;
        }
        if(start.after(end)){
            javax.swing.JOptionPane.showMessageDialog(mainPanel, 
                    "End date cannot be before start date.");
            return;
        }
        

        RetrieveValues task = new RetrieveValues(
                selected,start,end);
                
        task.addPropertyChangeListener(this);
        task.execute();
       
        
    }
    
    private void selectedStreamsPopulated(List<VegaStreamContainer> populated){
        
        try{
            if(lastOperation.equalsIgnoreCase("plotselected")){
                plotSelected(populated);
                lastOperation = "unknown";
            }else if(lastOperation.equalsIgnoreCase("retrieve")){
                //do nothing;
                lastOperation = "unknown";
            }else if(lastOperation.equalsIgnoreCase("save")){
                IDateFormatter format;
                if(outputDateFormatCB.getSelectedIndex()<=2){
                    format = new JavaSimpleDateFormat((String)outputDateFormatCB.getSelectedItem());
                }else if(outputDateFormatCB.getSelectedIndex()==3){
                    format = new SeparateColumnDateFormat(",");
                }else if(outputDateFormatCB.getSelectedIndex()==4){
                    format = new YearFractionDateFormat();
                }else if(outputDateFormatCB.getSelectedIndex()==5){
                    format = new JulianDayDateFormat();
                }else{
                    format = new JavaSimpleDateFormat("yyyy/MM/dd HH:mm:ss");
                }
                saveAsCsv(createSparseMatrix(populated,null),format);
            }else if(lastOperation.equalsIgnoreCase("aggregate")){
                IDateFormatter format;
                if(outputDateFormatCB.getSelectedIndex()<=2){
                    format = new JavaSimpleDateFormat((String)outputDateFormatCB.getSelectedItem());
                }else if(outputDateFormatCB.getSelectedIndex()==3){
                    format = new SeparateColumnDateFormat(",");
                }else if(outputDateFormatCB.getSelectedIndex()==4){
                    format = new YearFractionDateFormat();
                }else if(outputDateFormatCB.getSelectedIndex()==5){
                    format = new JulianDayDateFormat();
                }else{
                    format = new JavaSimpleDateFormat("yyyy/MM/dd HH:mm:ss");
                }
                saveAsCsv(createSparseMatrix(populated,new CSIStyleAggregator()),format);
            }else{
                //do nothing
            }
        
        }catch(Exception e){
            e.printStackTrace();
        }
    }

    private void plotSelected(List<VegaStreamContainer> selected){
        
        while(dataset.getSeriesCount() > 0){
            dataset.removeSeries(dataset.getSeriesKey(0));
        }
        
        for(VegaStreamContainer v:selected){
            if(v.values == null){
                continue;
            }
            
            double[][] tmp = new double[2][];

            double[] dates = new double[v.datetime.size()];
            for(int i=0;i<v.datetime.size();i++){
                dates[i] = (double)v.datetime.get(i).getTime();
            }

            tmp[0] = dates;
            double[] values = new double[v.values.size()];
            for(int i=0;i<v.datetime.size();i++){
                //This filters out any error derived outliers for the preview graph
                // errors are still included in the csv output.
                if(v.values.get(i)>-6000){
                    values[i] = v.values.get(i);
                }
            }
            tmp[1] = values;

            dataset.addSeries(v.toString(), tmp);
            
        }

    }
    
    // Variables declaration - do not modify//GEN-BEGIN:variables
    private javax.swing.JButton addButton;
    private javax.swing.JTextField aggSpanTF;
    private javax.swing.JComboBox aggTypeCB;
    private javax.swing.JPanel chartPanel;
    private javax.swing.JTextField endDateTF;
    private javax.swing.JLabel fileTypeLabel;
    private javax.swing.JTextField filterField;
    private javax.swing.JLabel jLabel1;
    private javax.swing.JLabel jLabel2;
    private javax.swing.JLabel jLabel3;
    private javax.swing.JLabel jLabel4;
    private javax.swing.JLabel jLabel5;
    private javax.swing.JLabel jLabel6;
    private javax.swing.JLabel jLabel7;
    private javax.swing.JLabel jLabel8;
    private javax.swing.JPanel jPanel1;
    private javax.swing.JPanel jPanel2;
    private javax.swing.JPanel jPanel3;
    private javax.swing.JScrollPane jScrollPane1;
    private javax.swing.JScrollPane jScrollPane2;
    private javax.swing.JPanel mainPanel;
    private javax.swing.JMenuBar menuBar;
    private javax.swing.JLabel metadataLabel;
    private javax.swing.JComboBox outputDateFormatCB;
    private javax.swing.JButton previewButton;
    private javax.swing.JProgressBar progressBar;
    private javax.swing.JButton removeButton;
    private javax.swing.JList selectedStreamsList;
    private javax.swing.JComboBox sitesCB;
    private javax.swing.JTextField startDateTF;
    private javax.swing.JLabel statusAnimationLabel;
    private javax.swing.JLabel statusMessageLabel;
    private javax.swing.JPanel statusPanel;
    private javax.swing.JList streamsList;
    private javax.swing.JButton toCSVButton;
    private javax.swing.JComboBox variablesCB;
    // End of variables declaration//GEN-END:variables


    private void allStreamInfoRetrieved(List<VegaStreamContainer> streams){
        allStreamInfo = streams;
        shownStreams = streams;
        streamsList.setListData(streams.toArray(new VegaStreamContainer[0]));

        sitesCB.setEnabled(true);
    }
    
    public void actionPerformed(ActionEvent e) {
        if(e.getSource().equals(sitesCB)||e.getSource().equals(variablesCB)){
            if(e.getActionCommand().compareToIgnoreCase("comboboxchanged")==0){
                filterStreams();
            }
            
        }
    }
    
    public void filterStreams(){
        String site = (String)sitesCB.getSelectedItem();
        String variable = (String)variablesCB.getSelectedItem();
        String freeTextFilter = filterField.getText();
        boolean filterText = false;
        boolean filterSite = false;
        boolean filterVariable = false;
        
        if(allStreamInfo == null || site==null){
            return;
        }
        
        /*if((site.equalsIgnoreCase(""))){
        streamsList.setListData(allStreamInfo.toArray(new VegaStreamContainer[0]));
        shownStreams = allStreamInfo;
        return;
        }*/

        List<VegaStreamContainer> filtered = new ArrayList<VegaStreamContainer>();

        if(freeTextFilter != null && freeTextFilter.length() > 0){
            filterText = true;
        }

        if(site != null && !site.equalsIgnoreCase("")){
            filterSite = true;
        }

        if(variable != null && !variable.equalsIgnoreCase("")){
            filterVariable=true;
        }

        for(VegaStreamContainer v:allStreamInfo){

            //don't display any that are 'selected'. Those are in the selected list
            if(v.selected){
                continue;
            }

            if(filterText &&
                    !v.toString().toLowerCase().contains(freeTextFilter.toLowerCase())){
                continue;
            }

            if(filterSite && !v.site.equalsIgnoreCase(site)){
                continue;
            }

            if(filterVariable && !v.variable.equalsIgnoreCase(variable)){
                continue;
            }

            filtered.add(v);
        }

        
        streamsList.setListData(filtered.toArray(new VegaStreamContainer[0]));
        shownStreams = filtered;
    }
    
    
    private class RetrieveValues extends SwingWorker<List<VegaStreamContainer>,Integer>{

        List<VegaStreamContainer> streams;
        Date start;
        Date end;
        
        public RetrieveValues(List<VegaStreamContainer> streams, Date start, Date end){
            this.streams = streams;
            this.start = start;
            this.end = end;
        }
        
        @Override
        protected List<VegaStreamContainer> doInBackground() throws Exception {
            //This may be excessive as it is passed by reference and 
            // we could probably just assume the passed objects have been filled
            // but we'll return the filled objects anyway.
            
            ArrayList<VegaStreamContainer> populated 
                    = new ArrayList<VegaStreamContainer>();
            VegaStreamContainer tmp;
            
            for(int i=0;i<streams.size();i++){
                tmp = streams.get(i);
                if(tmp.startRetrieved.compareTo(start)==0 &&
                        tmp.endRetrieved.compareTo(end)==0 &&
                        !tmp.hasNoValues){

                    //If the dates are the same and the object contains values
                    // do not re-retrieve
                    populated.add(tmp);
                    continue;
                }
                tmp.startRetrieved = start;
                tmp.endRetrieved = end;
                populateValues(tmp);
                populated.add(tmp);
                this.setProgress((100*i)/streams.size());
            }
            return populated;
            
        }
        
        @Override
        protected void done(){
            try{
                selectedStreamsPopulated(get());
            }catch(Exception ignore){
            }
            
        }
        
        private boolean populateValues(VegaStreamContainer stream)throws Exception{

            source.getData(stream);
            
            return true;

            /*

            try{
                
            
            SimpleDateFormat simp = new SimpleDateFormat("yyyy/MM/dd");
            
            PreparedStatement retrieveValues = conn.prepareStatement(
                    "SELECT * FROM `Values` WHERE StreamID = ? AND " +
                    " `DateTime` >=' " +
                    simp.format(start)+"'" +
                    " AND `DateTime` <= '"+
                    simp.format(end) + "' ORDER BY `DateTime`");
            
            retrieveValues.setInt(1,stream.streamID);
            
            ResultSet rs = retrieveValues.executeQuery();
            //let's see if this makes it more memory efficient. It isn't used anywhere
            //stream.retrieved = rs;
            
            
            ArrayList<Date> dateTime = new ArrayList<Date>();
            ArrayList<Double> values = new ArrayList<Double>();
            //ArrayList<String> flags = new ArrayList<String>();

            while(rs.next()){
                dateTime.add(new Date(rs.getTimestamp("DateTime").getTime()));
                values.add(rs.getDouble("Value"));
                //flags.add(rs.getString("Flag"));
            }
            
            rs.close();
            retrieveValues.close();
            rs = null;
            retrieveValues = null;
            
            stream.datetime = dateTime;
            stream.values = values;
            //stream.flags = flags;
              
                   
            
            }catch(Exception e){
                e.printStackTrace();
            }finally{
                
            }
             return true;

             */
        }
             

        
    }

    
    private class UpdateValues extends SwingWorker<Integer,Integer>{


        SparseMatrixContainer toParse;
        int updated = 0;
        int deleted = 0;
        int inserted = 0;
        int ignored = 0;
        
        public UpdateValues(SparseMatrixContainer toParse){
            this.toParse = toParse;
        }
        
        @Override
        protected Integer doInBackground() throws Exception {
        try{
            
        
            Pattern id = Pattern.compile(".*streamID:(\\d+).*",Pattern.DOTALL );
            
            int[] streamids = new int[toParse.headers.length -1];
            
            for(int i=0;i<streamids.length;i++){
                Matcher match = id.matcher(toParse.headers[i+1]);
               
                if( match.matches() & match.group(1)!= null){
                    streamids[i] = Integer.parseInt(match.group(1));
                }else{
                    throw new Exception("Stream ID cannot be found in header.");
                }
            }
            
            int estimatedNumber = toParse.values.length * (toParse.headers.length-1);
            int estimatedNumberDone = 0;
            
            for(int i=0;i<toParse.values.length;i++){
                for(int j=0;j<toParse.values[i].length;j++){
                    String result = actOnValue(toParse.values[i][j],toParse.datetime[i],streamids[j]);
                    
                    if(result.compareTo("inserted")==0){
                        inserted++;
                    }else if(result.compareTo("deleted")==0){
                        deleted++;
                    }else if(result.compareTo("updated") ==0){
                        updated++;
                    }else{
                        ignored++;
                    }
                    estimatedNumberDone++;
                    this.setProgress((100*estimatedNumberDone)/estimatedNumber);
                }
            }
        }catch(Exception e){
            e.printStackTrace();
        }

            return 0;//this doesn't actually mean anything
            
        }
        
        @Override
        protected void done(){
            dbUpdateDone(inserted,deleted,updated,ignored);
        }
        
        SimpleDateFormat simp = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
        
        private String actOnValue(double val,Date datetime,int streamid) throws SQLException{
            
            String actTaken = "ignored";
            boolean valExistsInDb = false;

            //Statement sel = conn.createStatement();
            Statement sel = null;
            
            String sql = "SELECT `Value` FROM `Values`" +
                    " WHERE StreamID='" + streamid + "' AND `DateTime` = '" + simp.format(datetime) + "'";
                    
            ResultSet rs = sel.executeQuery(sql);
            double dbValue = Double.NaN;
            if(rs.next()){
                dbValue = rs.getDouble("Value");
                valExistsInDb = true;
            }else{
                valExistsInDb = false;
            }
            rs.close();
            
            if(valExistsInDb & Double.isNaN(val)){
                //need to be deleting the value in the database.
                actTaken = "deleted";
                
                sel.executeUpdate("DELETE FROM `Values` WHERE " +
                        " StreamID='" + streamid + "' AND `DateTime`='" + simp.format(datetime) + "'");
                
            }else if(!valExistsInDb & !Double.isNaN(val)){
                //need to be inserting the new value.
                actTaken = "inserted";
                
                sel.executeUpdate("INSERT INTO `Values`(Value,`DateTime`,StreamID,UTCOffset) " +
                        " VALUES('" + val + "','"+ simp.format(datetime) +"','"+ streamid + "','-6')");
                        
                
            }else if(valExistsInDb & Double.compare(val,dbValue)!=0){
                //need to be updating
                actTaken = "updated";
                
                sel.executeUpdate("UPDATE `Values` SET `Value` = '" + val +"'" +
                        " WHERE `DateTime`='" + simp.format(datetime) + "' AND " +
                        " StreamID = '" + streamid + "'");
                
            }else if(valExistsInDb & Double.compare(val,dbValue)==0){
                 //do nothing, value must be the same
                actTaken = "ignored";
            }else{
                //not sure, assume we'll ignore it.
               actTaken = "ignored";
            }
            
            sel.close();
            
            return actTaken;
        }
        
    }

    
}
